{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "EgTCZqlhfS7i"
   },
   "source": [
    "# Installation of Pytorch\n",
    "Pytorch binary can be installed by pip, or complied following instructions in https://github.com/pytorch/pytorch#from-source.\n",
    "\n",
    "Here we will use pip."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define a Set of Computation Nodes\n",
    "\n",
    "| &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Name  &nbsp;&nbsp;&nbsp;&nbsp;  &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;    |   &nbsp;&nbsp; &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Fomula &nbsp;&nbsp;&nbsp;&nbsp;   &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;   &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;    |   &nbsp;&nbsp;  Gradients  |\n",
    "|:-------------:|:------------- |:----- |\n",
    "| Linear      | $y=x^T\\cdot W+b$ | $\\frac{\\partial \\mathcal{L}}{\\partial x}=W\\cdot\\frac{\\partial \\mathcal{L}}{\\partial y}\\\\\\frac{\\partial \\mathcal{L}}{\\partial W}=x^T\\cdot\\frac{\\partial \\mathcal{L}}{\\partial y}\\\\\\frac{\\partial \\mathcal{L}}{\\partial b}=\\frac{\\partial \\mathcal{L}}{\\partial y}$ |\n",
    "| Sigmoid     | $y=\\frac{1}{1+e^{-x}}$  | $\\frac{\\partial \\mathcal{L}}{\\partial x}=\\frac{\\partial \\mathcal{L}}{\\partial y}(1-y)y$ |\n",
    "| Softmax     | $y_j=\\frac{e^{x_j}}{\\sum\\limits_i e^{x_i}}$ | $\\frac{\\partial \\mathcal{L}}{\\partial x_j}=\\frac{\\partial \\mathcal{L}}{\\partial y_j}y_j-y_j\\sum\\limits_i \\frac{\\partial \\mathcal{L}}{\\partial y_i}y_i$ |\n",
    "| CrossEntropy | $y=-\\sum\\limits_i p_i \\log(x_i)$ | $\\frac{\\partial \\mathcal{L}}{\\partial x_i}=-\\frac{\\partial \\mathcal{L}}{\\partial y}\\frac{p_i}{x_i}$ |\n",
    "| Mean  | $y=\\frac{1}{N}\\sum\\limits_i x_i$ | $\\frac{\\partial \\mathcal{L}}{\\partial x_i}=\\frac{1}{N}\\frac{\\partial \\mathcal{L}}{\\partial y}$ |"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 136,
     "output_extras": [
      {
       "item_id": 5
      },
      {
       "item_id": 8
      }
     ]
    },
    "colab_type": "code",
    "collapsed": true,
    "executionInfo": {
     "elapsed": 86779,
     "status": "ok",
     "timestamp": 1519172965534,
     "user": {
      "displayName": "li li",
      "photoUrl": "//lh5.googleusercontent.com/-EpOiByGU5dA/AAAAAAAAAAI/AAAAAAAAACc/ZQHNZAF7xMQ/s50-c-k-no/photo.jpg",
      "userId": "118127190614979245201"
     },
     "user_tz": -60
    },
    "id": "qc3aNHrlNUgb",
    "outputId": "3049d2c1-7057-4e93-da9a-8c48092244c6"
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    import torch\n",
    "except:\n",
    "    !pip install http://download.pytorch.org/whl/cu80/torch-0.3.0.post4-cp36-cp36m-linux_x86_64.whl\n",
    "    import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchvision import datasets, transforms\n",
    "\n",
    "# data loader: we split the dataset into training set and test set.\n",
    "batch_size = 200\n",
    "train_loader = torch.utils.data.DataLoader(\n",
    "    datasets.MNIST('../data', train=True, download=True,\n",
    "                   transform=transforms.Compose([\n",
    "                       transforms.ToTensor(),\n",
    "                       transforms.Normalize((0.1307,), (0.3081,))\n",
    "                   ])),\n",
    "    batch_size=batch_size, shuffle=True)\n",
    "test_loader = torch.utils.data.DataLoader(\n",
    "    datasets.MNIST('../data', train=False, transform=transforms.Compose([\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize((0.1307,), (0.3081,))\n",
    "    ])),\n",
    "    batch_size=batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Net(\n",
      "  (linear): Linear(in_features=784, out_features=10, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "# create a neural network\n",
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.linear = nn.Linear(28 * 28, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        return F.log_softmax(self.linear(x))\n",
    "\n",
    "\n",
    "net = Net()\n",
    "print(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch.optim as optim\n",
    "\n",
    "def train(learning_rate=0.5, epochs=10):\n",
    "\n",
    "    # create a stochastic gradient descent optimizer\n",
    "    optimizer = optim.SGD(net.parameters(), lr=learning_rate, momentum=0)\n",
    "    # create a loss function\n",
    "    loss_func = nn.NLLLoss()\n",
    "\n",
    "    # run the main training loop\n",
    "    for epoch in range(epochs):\n",
    "        for batch_idx, (data, target) in enumerate(train_loader):\n",
    "            # resize data from (batch_size, 1, 28, 28) to (batch_size, 28*28)\n",
    "            data = data.view(-1, 28 * 28)\n",
    "            optimizer.zero_grad()\n",
    "            net_out = net(data)\n",
    "            loss = loss_func(net_out, target)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "        print('Train Epoch: {} \\tLoss: {:.6f}'.format(epoch, loss.data[0]))\n",
    "\n",
    "    # run a test loop\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    for data, target in test_loader:\n",
    "        data = data.view(-1, 28 * 28)\n",
    "        net_out = net(data)\n",
    "        # sum up batch loss\n",
    "        test_loss += loss_func(net_out, target).data[0]\n",
    "        # get the index of the max log-probability\n",
    "        pred = net_out.data.max(1)[1]\n",
    "        correct += pred.eq(target.data).sum()\n",
    "\n",
    "    test_loss /= len(test_loader.dataset)\n",
    "    print('\\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\\n'.format(\n",
    "        test_loss, correct, len(test_loader.dataset),\n",
    "        100. * correct / len(test_loader.dataset)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/lili/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:11: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  # This is added back by InteractiveShellApp.init_path()\n",
      "/Users/lili/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:20: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 0 \tLoss: 0.418251\n",
      "Train Epoch: 1 \tLoss: 0.389646\n"
     ]
    }
   ],
   "source": [
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "default_view": {},
   "name": "nice.ipynb",
   "provenance": [
    {
     "file_id": "1Lpp0qZjeF-NK_S6ROzsNvQzd2fFQ0Akt",
     "timestamp": 1517758638516
    }
   ],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
